# -*- coding: utf-8 -*-

BOT_NAME = 'SinaWeiboSpider'

SPIDER_MODULES = ['SinaWeiboSpider.spiders']
NEWSPIDER_MODULE = 'SinaWeiboSpider.spiders'


ROBOTSTXT_OBEY = False

CONCURRENT_REQUESTS = 16

DOWNLOAD_DELAY = 0.1

COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
# TELNETCONSOLE_ENABLED = False

# Override the default request headers:
DEFAULT_REQUEST_HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:61.0) Gecko/20100101 Firefox/61.0',
    'Cookie': '_T_WM=98425059843; SSOLoginState=1585050871; SUB=_2A25zfYSnDeRhGeNK71AV8y3Mwz2IHXVQgSzvrDV6PUJbktANLUHBkW1NSX0x7z9pxh2QleV5zxgrN9CiRtiu5jiX; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WhJ4ghp97xQ9zYjbSLg-nod5JpX5KzhUgL.Fo-XShzXe0e71h22dJLoIp7LxK-LB.eLBK5LxKBLB.2L1-2LxK.LBKeL1--0; SUHB=0RyqJ2Is8cV3MT; XSRF-TOKEN=b24418; WEIBOCN_FROM=1110006030; MLOGIN=1; M_WEIBOCN_PARAMS=luicode%3D20000174%26uicode%3D20000174',
}


# SPIDER_MIDDLEWARES = {
#    'SinaWeiboSpider.middlewares.SinaweibospiderSpiderMiddleware': 543,
# }

# REDIRECT_ENABLED = False

DOWNLOADER_MIDDLEWARES = {
    # 'SinaWeiboSpider.middlewares.CookieMiddleware': 300,
    'SinaWeiboSpider.middlewares.RedirectMiddleware': 200,
}


ITEM_PIPELINES = {
   'SinaWeiboSpider.pipelines.MongoPipline': 300,
}

# mongodb配置
MONGO_URI = 'mongodb://127.0.0.1:27017/'
DB_NAME = 'SinaWeibo'

# Redis 配置
# REDIS_HOST = '127.0.0.1'
# REDIS_PORT = 6379
#
# DOWNLOAD_TIMEOUT = 10
#
# RETRY_TIMES = 1
#
# # 去重的配置：
# DUPEFILTER_CLASS = 'scrapy_redis.dupefilter.RFPDupeFilter'
#
# # 调度器配置：
# SCHEDULER = "scrapy_redis.scheduler.Scheduler"
#
# SCHEDULER_QUEUE_CLASS = 'scrapy_redis.queue.PriorityQueue'  # 默认使用优先级队列（默认），其他：PriorityQueue（有序集合），FifoQueue（列表）、LifoQueue（列表）
# SCHEDULER_QUEUE_KEY = '%(spider)s:requests'  # 调度器中请求存放在redis中的key
# SCHEDULER_DUPEFILTER_KEY = '%(spider)s:dupefilter'  # 去重规则，在redis中保存时对应的key
# SCHEDULER_SERIALIZER = "scrapy_redis.picklecompat"  # 对保存到redis中的数据进行序列化，默认使用pickle
#
# SCHEDULER_PERSIST = True  # 是否在关闭时候保留原来的调度器和去重记录，True=保留，False=清空
# SCHEDULER_FLUSH_ON_START = False  # 是否在开始之前清空 调度器和去重记录，True=清空，False=不清空


